Augment dylib search paths for rustc/build scripts
authorAlex Crichton <alex@alexcrichton.com>
Mon, 19 Jan 2015 23:46:07 +0000 (15:46 -0800)
committerAlex Crichton <alex@alexcrichton.com>
Mon, 19 Jan 2015 23:48:06 +0000 (15:48 -0800)
Whenever a build script or a plugin depends on a native dynamic library, the
dylib search path will be required to include the path to the generated dynamic
library. Currently Cargo does not augment the search path, causing load errors
or execution errors whenever builds are attempted in this case.

This commit remedies the situation by ensuring that dynamic libraries show up in
dylib search paths. This commit is necessary to fully migrate the remaining
tests from the old build command infrastructure to build scripts.

Cargo.toml
src/cargo/ops/cargo_rustc/custom_build.rs
src/cargo/ops/cargo_rustc/engine.rs
src/cargo/ops/cargo_rustc/mod.rs
tests/test_cargo_compile_custom_build.rs
tests/test_cargo_compile_plugins.rs

index 8ff3ffba1f27fbe5795dc20e9bd264137598b890..d6215a66beda68a9530ffdeaadfbeaebfeee2933 100644 (file)
@@ -5,11 +5,6 @@ authors = ["Yehuda Katz <wycats@gmail.com>",
            "Carl Lerche <me@carllerche.com>",
            "Alex Crichton <alex@alexcrichton.com>"]
 
-[profile.test]
-debug = false
-[profile.dev]
-debug = false
-
 [lib]
 name = "cargo"
 path = "src/cargo/lib.rs"
index 54b633d58eb5c56b45e3ff228c2bc00a28d05161..e5af16fb29f47182cbdd2fe192f27b10d8e9de3c 100644 (file)
@@ -4,7 +4,6 @@ use std::io::fs::PathExtensions;
 use std::io::{fs, USER_RWX, File};
 use std::str;
 use std::sync::Mutex;
-use std::sync::mpsc::Sender;
 
 use core::{Package, Target, PackageId, PackageSet};
 use util::{CargoResult, human, Human};
@@ -26,8 +25,10 @@ pub struct BuildOutput {
     pub metadata: Vec<(String, String)>,
 }
 
+pub type BuildMap = HashMap<(PackageId, Kind), BuildOutput>;
+
 pub struct BuildState {
-    pub outputs: Mutex<HashMap<(PackageId, Kind), BuildOutput>>,
+    pub outputs: Mutex<BuildMap>,
 }
 
 /// Prepares a `Work` that executes the target as a custom build script.
@@ -100,6 +101,7 @@ pub fn prepare(pkg: &Package, target: &Target, req: Platform,
     let id = pkg.get_package_id().clone();
     let all = (id.clone(), pkg_name.clone(), build_state.clone(),
                build_output.clone());
+    let plugin_deps = super::crawl_build_deps(cx, pkg, target, Kind::Host);
 
     try!(fs::mkdir_recursive(&cx.layout(pkg, Kind::Target).build(pkg), USER_RWX));
     try!(fs::mkdir_recursive(&cx.layout(pkg, Kind::Host).build(pkg), USER_RWX));
@@ -111,7 +113,7 @@ pub fn prepare(pkg: &Package, target: &Target, req: Platform,
     //
     // Note that this has to do some extra work just before running the command
     // to determine extra environment variables and such.
-    let work = move |: desc_tx: Sender<String>| -> CargoResult<()> {
+    let work = Work::new(move |desc_tx| {
         // Make sure that OUT_DIR exists.
         //
         // If we have an old build directory, then just move it into place,
@@ -124,7 +126,9 @@ pub fn prepare(pkg: &Package, target: &Target, req: Platform,
         }
 
         // For all our native lib dependencies, pick up their metadata to pass
-        // along to this custom build command.
+        // along to this custom build command. We're also careful to augment our
+        // dynamic library search path in case the build script depended on any
+        // native dynamic libraries.
         let mut p = p;
         {
             let build_state = build_state.outputs.lock().unwrap();
@@ -137,6 +141,7 @@ pub fn prepare(pkg: &Package, target: &Target, req: Platform,
                               Some(value.as_slice()));
                 }
             }
+            p = try!(super::add_plugin_deps(p, &*build_state, plugin_deps));
         }
 
         // And now finally, run the build command itself!
@@ -167,7 +172,7 @@ pub fn prepare(pkg: &Package, target: &Target, req: Platform,
         }));
 
         Ok(())
-    };
+    });
 
     // Now that we've prepared our work-to-do, we need to prepare the fresh work
     // itself to run when we actually end up just discarding what we calculated
@@ -181,7 +186,7 @@ pub fn prepare(pkg: &Package, target: &Target, req: Platform,
     let (freshness, dirty, fresh) =
             try!(fingerprint::prepare_build_cmd(cx, pkg, kind, Some(target)));
     let dirty = Work::new(move |tx| {
-        try!(work(tx.clone()));
+        try!(work.call((tx.clone())));
         dirty.call(tx)
     });
     let fresh = Work::new(move |tx| {
index e810581398f9242e2d02ea2b5d0532e8da3578de..366753c82a41d84f8295e893ded5623f6c7ab6cd 100644 (file)
@@ -2,6 +2,7 @@ use std::collections::HashMap;
 use std::ffi::CString;
 use std::fmt::{self, Formatter};
 use std::io::process::ProcessOutput;
+use std::os;
 use std::path::BytesContainer;
 
 use util::{self, CargoResult, ProcessError, ProcessBuilder};
@@ -84,6 +85,12 @@ impl CommandPrototype {
         self
     }
 
+    pub fn get_env(&self, var: &str) -> Option<CString> {
+        self.env.get(var).cloned().or_else(|| {
+            Some(os::getenv(var).map(|s| CString::from_vec(s.into_bytes())))
+        }).and_then(|val| val)
+    }
+
     pub fn get_envs(&self) -> &HashMap<String, Option<CString>> {
         &self.env
     }
index a5d56339813057d613b2e5df61e8768af0bb5389..d50bc0edee4a283560538eb951acadf66bd72bb2 100644 (file)
@@ -1,6 +1,8 @@
 use std::collections::{HashSet, HashMap};
 use std::dynamic_lib::DynamicLibrary;
+use std::ffi::CString;
 use std::io::fs::{self, PathExtensions};
+use std::os;
 use std::path;
 use std::sync::Arc;
 
@@ -16,7 +18,7 @@ pub use self::context::Context;
 pub use self::context::Platform;
 pub use self::engine::{CommandPrototype, CommandType, ExecEngine, ProcessEngine};
 pub use self::layout::{Layout, LayoutProxy};
-pub use self::custom_build::BuildOutput;
+pub use self::custom_build::{BuildOutput, BuildMap};
 
 mod context;
 mod compilation;
@@ -333,7 +335,9 @@ fn rustc(package: &Package, target: &Target,
     let crate_types = target.rustc_crate_types();
     let rustcs = try!(prepare_rustc(package, target, crate_types, cx, req));
 
-    rustcs.into_iter().map(|(rustc, kind)| {
+    let plugin_deps = crawl_build_deps(cx, package, target, Kind::Host);
+
+    return rustcs.into_iter().map(|(rustc, kind)| {
         let name = package.get_name().to_string();
         let is_path_source = package.get_package_id().get_source_id().is_path();
         let show_warnings = package.get_package_id() == cx.resolve.root() ||
@@ -346,37 +350,12 @@ fn rustc(package: &Package, target: &Target,
 
         // Prepare the native lib state (extra -L and -l flags)
         let build_state = cx.build_state.clone();
-        let mut native_lib_deps = HashSet::new();
         let current_id = package.get_package_id().clone();
-
+        let plugin_deps = plugin_deps.clone();
+        let mut native_lib_deps = crawl_build_deps(cx, package, target, kind);
         if package.has_custom_build() && !target.get_profile().is_custom_build() {
-            native_lib_deps.insert(current_id.clone());
+            native_lib_deps.insert(0, current_id.clone());
         }
-        // Visit dependencies transitively to figure out what our native
-        // dependencies are (for -L and -l flags).
-        //
-        // Be sure to only look at targets of the same Kind, however, as we
-        // don't want to include native libs of plugins for targets for example.
-        fn visit<'a>(cx: &'a Context, pkg: &'a Package, target: &Target,
-                     kind: Kind,
-                     visiting: &mut HashSet<&'a PackageId>,
-                     libs: &mut HashSet<PackageId>) {
-            for &(pkg, target) in cx.dep_targets(pkg, target).iter() {
-                let req = cx.get_requirement(pkg, target);
-                if !req.includes(kind) { continue }
-                if !visiting.insert(pkg.get_package_id()) { continue }
-
-                if pkg.has_custom_build() {
-                    libs.insert(pkg.get_package_id().clone());
-                }
-                visit(cx, pkg, target, kind, visiting, libs);
-                visiting.remove(&pkg.get_package_id());
-            }
-        }
-        visit(cx, package, target, kind,
-              &mut HashSet::new(), &mut native_lib_deps);
-        let mut native_lib_deps = native_lib_deps.into_iter().collect::<Vec<_>>();
-        native_lib_deps.sort();
 
         // If we are a binary and the package also contains a library, then we
         // don't pass the `-l` flags.
@@ -392,21 +371,15 @@ fn rustc(package: &Package, target: &Target,
             let mut rustc = rustc;
 
             // Only at runtime have we discovered what the extra -L and -l
-            // arguments are for native libraries, so we process those here.
-            {
-                let build_state = build_state.outputs.lock().unwrap();
-                for id in native_lib_deps.into_iter() {
-                    let output = &build_state[(id.clone(), kind)];
-                    for path in output.library_paths.iter() {
-                        rustc = rustc.arg("-L").arg(path);
-                    }
-                    if pass_l_flag && id == current_id {
-                        for name in output.library_links.iter() {
-                            rustc = rustc.arg("-l").arg(name.as_slice());
-                        }
-                    }
-                }
-            }
+            // arguments are for native libraries, so we process those here. We
+            // also need to be sure to add any -L paths for our plugins to the
+            // dynamic library load path as a plugin's dynamic library may be
+            // located somewhere in there.
+            let build_state = build_state.outputs.lock().unwrap();
+            rustc = add_native_deps(rustc, &*build_state, native_lib_deps,
+                                    kind, pass_l_flag, &current_id);
+            rustc = try!(add_plugin_deps(rustc, &*build_state, plugin_deps));
+            drop(build_state);
 
             // FIXME(rust-lang/rust#18913): we probably shouldn't have to do
             //                              this manually
@@ -428,7 +401,76 @@ fn rustc(package: &Package, target: &Target,
             Ok(())
 
         }), kind))
-    }).collect()
+    }).collect();
+
+    // Add all relevant -L and -l flags from dependencies (now calculated and
+    // present in `state`) to the command provided
+    fn add_native_deps(mut rustc: CommandPrototype,
+                       build_state: &BuildMap,
+                       native_lib_deps: Vec<PackageId>,
+                       kind: Kind,
+                       pass_l_flag: bool,
+                       current_id: &PackageId) -> CommandPrototype {
+        for id in native_lib_deps.into_iter() {
+            let output = &build_state[(id.clone(), kind)];
+            for path in output.library_paths.iter() {
+                rustc = rustc.arg("-L").arg(path);
+            }
+            if pass_l_flag && id == *current_id {
+                for name in output.library_links.iter() {
+                    rustc = rustc.arg("-l").arg(name.as_slice());
+                }
+            }
+        }
+        return rustc;
+    }
+}
+
+fn crawl_build_deps<'a>(cx: &'a Context, pkg: &'a Package,
+                        target: &Target, kind: Kind) -> Vec<PackageId> {
+    let mut deps = HashSet::new();
+    visit(cx, pkg, target, kind, &mut HashSet::new(), &mut deps);
+    let mut ret: Vec<_> = deps.into_iter().collect();
+    ret.sort();
+    return ret;
+
+    fn visit<'a>(cx: &'a Context, pkg: &'a Package, target: &Target,
+                 kind: Kind,
+                 visiting: &mut HashSet<&'a PackageId>,
+                 libs: &mut HashSet<PackageId>) {
+        for &(pkg, target) in cx.dep_targets(pkg, target).iter() {
+            let req = cx.get_requirement(pkg, target);
+            if !req.includes(kind) { continue }
+            if !visiting.insert(pkg.get_package_id()) { continue }
+
+            if pkg.has_custom_build() {
+                libs.insert(pkg.get_package_id().clone());
+            }
+            visit(cx, pkg, target, kind, visiting, libs);
+            visiting.remove(&pkg.get_package_id());
+        }
+    }
+}
+
+// For all plugin dependencies, add their -L paths (now calculated and
+// present in `state`) to the dynamic library load path for the command to
+// execute.
+fn add_plugin_deps(rustc: CommandPrototype,
+                   build_state: &BuildMap,
+                   plugin_deps: Vec<PackageId>)
+                   -> CargoResult<CommandPrototype> {
+    let var = DynamicLibrary::envvar();
+    let search_path = rustc.get_env(var)
+                           .unwrap_or(CString::from_slice(b""));
+    let mut search_path = os::split_paths(search_path);
+    for id in plugin_deps.into_iter() {
+        let output = &build_state[(id, Kind::Host)];
+        for path in output.library_paths.iter() {
+            search_path.push(path.clone());
+        }
+    }
+    let search_path = try!(join_paths(&search_path[], var));
+    Ok(rustc.env(var, Some(search_path)))
 }
 
 fn prepare_rustc(package: &Package, target: &Target, crate_types: Vec<&str>,
index 6c6a7dd5df569b1875b5474760345a7841d6533d..361f41171a8029cb16b7b50ada3b05b38dfb356b 100644 (file)
@@ -1,4 +1,5 @@
-use std::io::File;
+use std::io::{File, fs};
+use std::os;
 
 use support::{project, execs, cargo_dir};
 use support::{COMPILING, RUNNING, DOCTEST};
@@ -1004,3 +1005,74 @@ test!(test_dev_dep_build_script {
 
     assert_that(p.cargo_process("test"), execs().with_status(0));
 });
+
+test!(build_script_with_dynamic_native_dependency {
+    let build = project("builder")
+        .file("Cargo.toml", r#"
+            [package]
+            name = "builder"
+            version = "0.0.1"
+            authors = []
+
+            [lib]
+            name = "builder"
+            crate-type = ["dylib"]
+        "#)
+        .file("src/lib.rs", r#"
+            #[no_mangle]
+            pub extern fn foo() {}
+        "#);
+    assert_that(build.cargo_process("build"),
+                execs().with_status(0).with_stderr(""));
+    let src = build.root().join("target");
+    let lib = fs::readdir(&src).unwrap().into_iter().find(|lib| {
+        let lib = lib.filename_str().unwrap();
+        lib.starts_with(os::consts::DLL_PREFIX) &&
+            lib.ends_with(os::consts::DLL_SUFFIX)
+    }).unwrap();
+    let libname = lib.filename_str().unwrap();
+    let libname = libname.slice(os::consts::DLL_PREFIX.len(),
+                                libname.len() - os::consts::DLL_SUFFIX.len());
+
+    let foo = project("foo")
+        .file("Cargo.toml", r#"
+            [package]
+            name = "foo"
+            version = "0.0.1"
+            authors = []
+            build = "build.rs"
+
+            [build-dependencies.bar]
+            path = "bar"
+        "#)
+        .file("build.rs", r#"
+            extern crate bar;
+            fn main() { bar::bar() }
+        "#)
+        .file("src/lib.rs", "")
+        .file("bar/Cargo.toml", r#"
+            [package]
+            name = "bar"
+            version = "0.0.1"
+            authors = []
+            build = "build.rs"
+        "#)
+        .file("bar/build.rs", r#"
+            use std::os;
+
+            fn main() {
+                let src = Path::new(os::getenv("SRC").unwrap());
+                println!("cargo:rustc-flags=-L {}", src.dir_path().display());
+            }
+        "#)
+        .file("bar/src/lib.rs", format!(r#"
+            pub fn bar() {{
+                #[link(name = "{}")]
+                extern {{ fn foo(); }}
+                unsafe {{ foo() }}
+            }}
+        "#, libname));
+
+    assert_that(foo.cargo_process("build").env("SRC", Some(lib.as_vec())),
+                execs().with_status(0));
+});
index 7379ea16e27ca015a135f5c76143943a98d3f21b..c246e854ff54049acc80a613b20d73f8f69b70ad 100644 (file)
@@ -94,18 +94,6 @@ test!(plugin_with_dynamic_native_dependency {
             name = "builder"
             crate-type = ["dylib"]
         "#)
-        .file("src/main.rs", r#"
-            #![allow(unstable)]
-            use std::io::fs;
-            use std::os;
-
-            fn main() {
-                let src = Path::new(os::getenv("SRC").unwrap());
-                let dst = Path::new(os::getenv("OUT_DIR").unwrap());
-                let dst = dst.join(src.filename().unwrap());
-                fs::rename(&src, &dst).unwrap();
-            }
-        "#)
         .file("src/lib.rs", r#"
             #[no_mangle]
             pub extern fn foo() {}
@@ -138,17 +126,26 @@ test!(plugin_with_dynamic_native_dependency {
 
             fn main() {}
         "#)
-        .file("bar/Cargo.toml", format!(r#"
+        .file("bar/Cargo.toml", r#"
             [package]
             name = "bar"
             version = "0.0.1"
             authors = []
-            build = '{}'
+            build = 'build.rs'
 
             [lib]
             name = "bar"
             plugin = true
-        "#, build.bin("builder").display()))
+        "#)
+        .file("bar/build.rs", r#"
+            use std::io::fs;
+            use std::os;
+
+            fn main() {
+                let src = Path::new(os::getenv("SRC").unwrap());
+                println!("cargo:rustc-flags=-L {}", src.dir_path().display());
+            }
+        "#)
         .file("bar/src/lib.rs", format!(r#"
             #![feature(plugin_registrar)]